# This section is a bunch of notes that I'm writing for myself. You needn't read them. Skip to the introduction.
# TODO: This will be the delegated notebook!
# FRI JAN 8 TODO:
# TODO: Create symlinks in this directory for the dataset so that it can be exported to other people more easily
# TODO: Clean up the names of things in this notebook
# TODO: Move all these functions to other files and create a delegation notebook to test them
# TODO: Create the standard deviation measurement from multiple translations, and create a visualization for that deviation in this notebook (with some tests, of course)
# class ViewConsistencyVarianceLoss: def __init__(self, tex_width, tex_height, num_labels, pyramid_weights=[1,1,1])
# def forward(self, scene_uvs, scene_translations)
# TODO: Smooth moving cube blender animation for demos
# FOR EXPERIMENT: With pure simulated data, like textured cube in blender that moves around, we could use mean squared error for measuring how good each method is!
# TODO: Figure out why the table is so blurry in the naive reconstructions. Is this because the MUNIT is randomly shifting the result image? It seems to be a discerete blur, in that a few shifts are averaged together...
# NOTE: it might be beneficial to use multiple values of recovery_resolution in the view consistency loss; because that way it can criticize both high and low detail scales. This can be done with multiple ViewConsistencyLoss objects; perhaps aggregated into a MultiScaleViewConsistencyLoss(nn.Module) class.
# NOTE: Uses of this might be for: reinforcement learning with multiple cameras, mobile robots, reinforcement learning with some temporal memory beetween frames and/or using optical flow. Might also be useful for data augmentation for image segmentation and object detection tasks?
# TODO: Once we get the thing working, can we then use the vid2vid to bake textures onto objects?
#TODO: Try using the crummy recovered textures from averaging the cyclegan outputs as the initial texture instead of random noise. Also try using a network that returns its own input as an initial neural networkk.
#IDEA: There's also a texture that gets evolved. The texture is pushed to match the output of teh translations, but the translations are only pushed to match the hue of that texture. That way it doesn't hold back the translation network **too** much but shuold remain non-blurry. Or better yet, the translation network is only responsible for shading and the texture does the rest...
#HOW TO INTEGRATE:
# turn the learned-neural-rendered-image-projection thing into a DataLoader class, and substitute that in for the current dataloader for the MUNIT algorihtm.
# then, just add the consistency loss. Do
#RENAMINGS: textures, weights becomes texture_pack, weight_pack
#RENAMINGS: num_labels becomes num_textures
#TODO: Define what a texture and scene are, with pictures ov UV scenes etc.
from rp import *
import torch
import icecream
#Install packages if needed:
pip_import('einops');
pip_import('torch' );
%config InlineBackend.figure_format='retina'
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
icecream.ic(device);
def display_images(images):
if isinstance(images,torch.Tensor):
images=as_numpy_images(images)
display_image(tiled_images(images))
def as_numpy_image(image):
if isinstance(image,np.ndarray):
return image.copy()
else:
return as_numpy_images(image.unsqueeze(0))[0]
def resize_images(images,size,interp='bilinear'):
return [cv_resize_image(image,size,interp) for image in images]
print("This is a photo: the target domain is an alphabet block")
photo_image=as_rgb_image(load_image(random_element(get_all_files('datasets/alphacube/photos'))))
icecream.ic(photo_image.shape)
display_image(photo_image)
print("This is a 'Scene': A picture of a 3d model's UV map (red/green), and blue 'label' channel indicating whats what")
scene_image=as_rgb_image(load_image(random_element(get_all_files('datasets/alphacube/scenes'))))
icecream.ic(scene_image.shape)
display_image(scene_image)
print("This is an example of a 'Texture', an image that gets applied to UV maps for a particular label. ")
print("This particular texture gets applied to the alphabet cube. Its a bit blurry because it was recovered from data.")
print("Note that in my code, we use square textures. This is an arbitrary choice; they don't have to be.")
display_image(load_image('assets/texture_example.png'))
print("The goal of this tutorial is to show you how some of the functions in this project are used, giving you")
print("a more visual intuition for this project as a whole")
image_paths=get_all_files('datasets/alphacube/scenes',sort_by='number')
image_paths=image_paths[:16] #For the previews, limit the number of samples. It makes the .ipynb files smaller.
cube_models=load_images(image_paths,show_progress=True,use_cache=True)
cube_models=[as_float_image(cube_model) for cube_model in cube_models]
cube_models=as_numpy_array(cube_models)
print("A random cube model:")
display_images(cube_models[:4])
stone='https://www.filterforge.com/filters/12449.jpg'
tiles='https://filterforge.com/filters/10857-v4.jpg'
wood ='https://filterforge.com/filters/8892.jpg'
paved='https://filterforge.com/filters/14157.jpg'
metal='https://www.filterforge.com/filters/1375.jpg'
gears='https://www.filterforge.com/filters/8624.jpg'
walls='https://www.filterforge.com/filters/15245.jpg'
grass='https://www.filterforge.com/filters/11635.jpg'
china='https://www.filterforge.com/filters/9935.jpg'
#Go ahead and modify this notebook here: choose your favorite two textures!
#The first one goes to the cube, and the second one goes to the table.
albedo =china
second_albedo=wood
albedo =load_image(albedo ,use_cache=True)
second_albedo=load_image(second_albedo,use_cache=True)
#Display the images:
ims=load_images([stone,tiles,wood,paved,metal,gears,walls,grass,china],use_cache=True)
ims=resize_images(ims,.25)
ims=labeled_images(ims,'stone,tiles,wood,paved,metal,gears,walls,grass,china'.split(','))
ims=tiled_images(ims)
print("Texture options:")
display_image(ims)
print("Albedo Map:")
display_image(albedo)
icecream.ic(albedo.shape)
print("Second Albedo Map:")
display_image(second_albedo)
icecream.ic(second_albedo.shape)
#Create the torch tensors:
torch_cube_models=as_torch_images(cube_models).to(device)
torch_albedo =torch.tensor(albedo ).to(device).permute(2,0,1)/255
torch_second_albedo=torch.tensor(second_albedo).to(device).permute(2,0,1)/255
from source.scene_reader import extract_scene_uvs_and_scene_labels
scene_uvs, scene_labels = extract_scene_uvs_and_scene_labels(torch_cube_models,[0,255])
icecream.ic(scene_labels.flatten().unique())
icecream.ic(torch_cube_models.shape,
scene_uvs .shape,
scene_labels .shape);
from source.projector import colorized_scene_labels
print("Colorized with arbitrary colors, such as blue and pink...")
colorized_labels = colorized_scene_labels(scene_labels, torch.Tensor([[1,0,.5],[0,.25,.5]]))
display_images(colorized_labels[:4])
print("Colorized with more arbitrary colors, such as black and green...")
colorized_labels = colorized_scene_labels(scene_labels, torch.Tensor([[0,1,0],[0,0,0]]))
display_images(colorized_labels[:4])
from source.projector import project_textures
textures=torch.stack((torch_albedo, torch_second_albedo))
icecream.ic(textures.shape)
scene_projections = project_textures(scene_uvs, scene_labels, textures)
print("Rendered images from torch: should look identical to the previous animation on every frame")
display_images(as_numpy_images(scene_projections[:4]))
from source.unprojector import unproject_translations, unproject_translations_individually
num_labels=len(textures)
recovery_resolution=1024
# recovery_resolution=512
# recovery_resolution=256
recovered_textures, _ = unproject_translations(scene_projections ,
scene_uvs ,
scene_labels ,
num_labels ,
output_height=recovery_resolution,
output_width =recovery_resolution)
print("Unprojection mean:")
display_images(recovered_textures)
w=torch.stack((_,_,_),dim=1)
w=w/w.max(dim=1,keepdim=True)[0].max(dim=2,keepdim=True)[0].max(dim=3,keepdim=True)[0]
print("Unprojection weights:")
display_images(w)
frames=[]
for scene_uv, scene_label, scene_projection, cube_model in zip(scene_uvs, scene_labels, scene_projections, cube_models):
# recovery_resolution=1024
recovery_resolution=512
# recovery_resolution=256
recovered_textures, _ = unproject_translations(scene_projection[None] ,
scene_uv [None] ,
scene_label [None] ,
num_labels ,
output_height=recovery_resolution,
output_width =recovery_resolution)
scene_projection = as_numpy_image (scene_projection )
recovered_textures = as_numpy_images(recovered_textures)
scene_width = get_image_width(scene_projection)
assert get_image_width(cube_model) == scene_width
scene_stuff = [scene_projection, cube_model]
scene_stuff = resize_images (scene_stuff, recovery_resolution/scene_width )
scene_stuff = labeled_images(scene_stuff, ['Scene Projection', 'UV Map and Labels'])
recovered_textures = labeled_images(recovered_textures, ['Recovered Albedo','Recovered Second Albedo'])
frame = grid_concatenated_images([recovered_textures, scene_stuff])
frames.append(frame)
display_image_slideshow(frames)